{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "excellent-clock",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import lxml.etree as le\n",
    "import pandas as pd\n",
    "import numpy as np\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "unique-father",
   "metadata": {},
   "outputs": [],
   "source": [
    "x = '//div[@id=\"leftcolumn\"]/a '#74\n",
    "y = '//div[@id=\"leftcolumn\"]//@href'\n",
    "url = 'https://www.runoob.com/html/html-tutorial.html'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "optimum-anchor",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "url = 'https://www.runoob.com/html/html-tutorial.html'\n",
    "\n",
    "x = '//div[@id=\"leftcolumn\"]/a/text()' #xpath代码\n",
    "content = requests.get(url = url).content\n",
    "\n",
    "contentx = le.HTML(content)\n",
    "\n",
    "# print(contentx)\n",
    "rets = contentx.xpath(x)\n",
    "# print(rets)\n",
    "data = []\n",
    "for category in rets:\n",
    "    data.append(category.strip())\n",
    "#     print(category.strip())\n",
    "data = data\n",
    "  \n",
    "#\n",
    "# import ktool # pip install ktool\n",
    "#\n",
    "# rets = ktool.xpath.xpath_all(content,x)\n",
    "#\n",
    "# for ret in rets:\n",
    "#     print(ret)\n",
    "#\n",
    "# a = ktool.xpath.xpath_union(content,x)\n",
    "# print(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "consolidated-postage",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>一级标题</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HTML 教程</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HTML 简介</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HTML 编辑器</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HTML 基础</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HTML 元素</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>HTML URL 编码</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>HTML 语言代码</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>HTTP 消息</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>HTTP 方法</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>键盘快捷键</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           一级标题\n",
       "0       HTML 教程\n",
       "1       HTML 简介\n",
       "2      HTML 编辑器\n",
       "3       HTML 基础\n",
       "4       HTML 元素\n",
       "..          ...\n",
       "69  HTML URL 编码\n",
       "70    HTML 语言代码\n",
       "71      HTTP 消息\n",
       "72      HTTP 方法\n",
       "73        键盘快捷键\n",
       "\n",
       "[74 rows x 1 columns]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "data = pd.DataFrame(data=data,\n",
    "                    columns=['一级标题'])\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "western-coupon",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.runoob.com/html/html-tutorial.html',\n",
       " 'https://www.runoob.com/html/html-intro.html',\n",
       " 'https://www.runoob.com/html/html-editors.html',\n",
       " 'https://www.runoob.com/html/html-basic.html',\n",
       " 'https://www.runoob.com/html/html-elements.html',\n",
       " 'https://www.runoob.com/html/html-attributes.html',\n",
       " 'https://www.runoob.com/html/html-headings.html',\n",
       " 'https://www.runoob.com/html/html-paragraphs.html',\n",
       " 'https://www.runoob.com/html/html-formatting.html',\n",
       " 'https://www.runoob.com/html/html-links.html',\n",
       " 'https://www.runoob.com/html/html-head.html',\n",
       " 'https://www.runoob.com/html/html-css.html',\n",
       " 'https://www.runoob.com/html/html-images.html',\n",
       " 'https://www.runoob.com/html/html-tables.html',\n",
       " 'https://www.runoob.com/html/html-lists.html',\n",
       " 'https://www.runoob.com/html/html-blocks.html',\n",
       " 'https://www.runoob.com/html/html-layouts.html',\n",
       " 'https://www.runoob.com/html/html-forms.html',\n",
       " 'https://www.runoob.com/html/html-iframes.html',\n",
       " 'https://www.runoob.com/html/html-colors.html',\n",
       " 'https://www.runoob.com/html/html-colornames.html',\n",
       " 'https://www.runoob.com/html/html-colorvalues.html',\n",
       " 'https://www.runoob.com/html/html-scripts.html',\n",
       " 'https://www.runoob.com/html/html-entities.html',\n",
       " 'https://www.runoob.com/html/html-url.html',\n",
       " 'https://www.runoob.com/html/html-quicklist.html',\n",
       " 'https://www.runoob.com/html/html-tag-name.html',\n",
       " 'https://www.runoob.com/html/html-summary.html',\n",
       " 'https://www.runoob.com/html/html-xhtml.html',\n",
       " 'https://www.runoob.com/html/html5-intro.html',\n",
       " 'https://www.runoob.comhtml5-browsers.html',\n",
       " 'https://www.runoob.com/html/html5-new-element.html',\n",
       " 'https://www.runoob.com/html/html5-canvas.html',\n",
       " 'https://www.runoob.com/html/html5-svg.html',\n",
       " 'https://www.runoob.comhtml5-mathml.html',\n",
       " 'https://www.runoob.com/html/html5-draganddrop.html',\n",
       " 'https://www.runoob.com/html/html5-geolocation.html',\n",
       " 'https://www.runoob.com/html/html5-video.html',\n",
       " 'https://www.runoob.com/html/html5-audio.html',\n",
       " 'https://www.runoob.com/html/html5-form-input-types.html',\n",
       " 'https://www.runoob.com/html/html5-form-elements.html',\n",
       " 'https://www.runoob.com/html/html5-form-attributes.html',\n",
       " 'https://www.runoob.com/html/html5-semantic-elements.html',\n",
       " 'https://www.runoob.com/html/html5-webstorage.html',\n",
       " 'https://www.runoob.comhtml5-web-sql.html',\n",
       " 'https://www.runoob.com/html/html5-app-cache.html',\n",
       " 'https://www.runoob.com/html/html5-webworkers.html',\n",
       " 'https://www.runoob.com/html/html5-serversentevents.html',\n",
       " 'https://www.runoob.com/html/html5-websocket.html',\n",
       " 'https://www.runoob.com/quiz/html5-quiz.html',\n",
       " 'https://www.runoob.com/html/html5-syntax.html',\n",
       " 'https://www.runoob.com/html/html-media.html',\n",
       " 'https://www.runoob.com/html/html-object.html',\n",
       " 'https://www.runoob.com/html/html-sounds.html',\n",
       " 'https://www.runoob.com/html/html-videos.html',\n",
       " 'https://www.runoob.com/html/html-examples.html',\n",
       " 'https://www.runoob.com/tags/html-reference.html',\n",
       " 'https://www.runoob.com/tags/ref-byfunc.html',\n",
       " 'https://www.runoob.com/tags/ref-standardattributes.html',\n",
       " 'https://www.runoob.com/tags/ref-eventattributes.html',\n",
       " 'https://www.runoob.com/tags/ref-canvas.html',\n",
       " 'https://www.runoob.com/tags/ref-av-dom.html',\n",
       " 'https://www.runoob.com/tags/html-elementsdoctypes.html',\n",
       " 'https://www.runoob.com/tags/html-colorname.html',\n",
       " 'https://www.runoob.com/tags/html-colorpicker.html',\n",
       " 'https://www.runoob.com/charsets/html-charsets.html',\n",
       " 'https://www.runoob.com/tags/html-ascii.html',\n",
       " 'https://www.runoob.com/tags/ref-entities.html',\n",
       " 'https://www.runoob.com/tags/html-symbols.html',\n",
       " 'https://www.runoob.com/tags/html-urlencode.html',\n",
       " 'https://www.runoob.com/tags/html-language-codes.html',\n",
       " 'https://www.runoob.com/tags/html-httpmessages.html',\n",
       " 'https://www.runoob.com/tags/html-httpmethods.html',\n",
       " 'https://www.runoob.com/tags/html-keyboardshortcuts.html']"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.runoob.com/html/html-tutorial.html'\n",
    "head = 'https://www.runoob.com'\n",
    "\n",
    "# x = '//div[@id=\"leftcolumn\"]/a/text()' #xpath代码\n",
    "content = requests.get(url = url).content\n",
    "\n",
    "contenty = le.HTML(content)\n",
    "\n",
    "# print(contentx)\n",
    "rets = contenty.xpath(y)\n",
    "# print(rets)\n",
    "datay = []\n",
    "for category in rets:\n",
    "    a = head + category.strip()\n",
    "    datay.append(a)\n",
    "datay\n",
    "#     print(category.strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "affecting-leisure",
   "metadata": {},
   "outputs": [],
   "source": [
    "datay = pd.DataFrame(data=datay,\n",
    "                    columns=['网址'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "married-prediction",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>一级标题</th>\n",
       "      <th>网址</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HTML 教程</td>\n",
       "      <td>https://www.runoob.com/html/html-tutorial.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HTML 简介</td>\n",
       "      <td>https://www.runoob.com/html/html-intro.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HTML 编辑器</td>\n",
       "      <td>https://www.runoob.com/html/html-editors.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HTML 基础</td>\n",
       "      <td>https://www.runoob.com/html/html-basic.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HTML 元素</td>\n",
       "      <td>https://www.runoob.com/html/html-elements.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>HTML URL 编码</td>\n",
       "      <td>https://www.runoob.com/tags/html-urlencode.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>HTML 语言代码</td>\n",
       "      <td>https://www.runoob.com/tags/html-language-code...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>HTTP 消息</td>\n",
       "      <td>https://www.runoob.com/tags/html-httpmessages....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>HTTP 方法</td>\n",
       "      <td>https://www.runoob.com/tags/html-httpmethods.html</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>键盘快捷键</td>\n",
       "      <td>https://www.runoob.com/tags/html-keyboardshort...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>74 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           一级标题                                                 网址\n",
       "0       HTML 教程     https://www.runoob.com/html/html-tutorial.html\n",
       "1       HTML 简介        https://www.runoob.com/html/html-intro.html\n",
       "2      HTML 编辑器      https://www.runoob.com/html/html-editors.html\n",
       "3       HTML 基础        https://www.runoob.com/html/html-basic.html\n",
       "4       HTML 元素     https://www.runoob.com/html/html-elements.html\n",
       "..          ...                                                ...\n",
       "69  HTML URL 编码    https://www.runoob.com/tags/html-urlencode.html\n",
       "70    HTML 语言代码  https://www.runoob.com/tags/html-language-code...\n",
       "71      HTTP 消息  https://www.runoob.com/tags/html-httpmessages....\n",
       "72      HTTP 方法  https://www.runoob.com/tags/html-httpmethods.html\n",
       "73        键盘快捷键  https://www.runoob.com/tags/html-keyboardshort...\n",
       "\n",
       "[74 rows x 2 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = pd.concat([data,datay],axis=1)\n",
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "demonstrated-promise",
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存cxcel文件\n",
    "a.to_excel('./xpath_homework.xls',\n",
    "          sheet_name='菜鸟教程爬取',\n",
    "          header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "deadly-cherry",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
